trendcrawler - Crawler - Neuer Ordner - SOURCE CODE

#!/usr/bin/perl

###########################
### Autor: Sebastian Enger / B.Sc. 
### Copyright: Sebastian Enger
### Licence: Commercial
### Version: 1.1.1  - 2013-09-18
### Contact: sebastian.enger@gmail.com
###########################

package Crawler::Basic;

use strict;
use LWP::UserAgent;
use HTTP::Request::Common qw(GET);
use HTTP::Cookies;
use LWP::Simple;
use HTTP::Status ();
use File::Path qw(make_path);
 
my %AlreadyScanned = ();
my $AlreadyScanned;

my $D;
my %D = ();

my $ScannedFile = "/home/thecerial/trendcrawler/logfile/alreadyscanned.txt";
my $CookieFile = "/home/thecerial/trendcrawler/logfile/cookies.txt";

# Konstruktor
sub new {
	my ($class) = @_;
	my $self = {
	#	'kontostand' => $inKontostand,
	#	'dispo' => $inDispo
	};
	bless $self, $class;
  make_path '/home/thecerial/trendcrawler/logfile/', {owner=>'thecerial', group=>'users'};
  return $self;
}

sub fetch(){

	my $self	= shift;
	my $url 	= shift;
	return if $url !~ m/^http/ig;
	
	my $max_retry = 30;
	my $sleep = 60;
	

... [truncated, 135 more lines] ...